import scrapy
from qsbk.items import QsbkItem

class QsbkSpiderSpider(scrapy.Spider):
    name = 'qsbk_spider'
    allowed_domains = ['qiushibaike.com']
    base_domain = 'https://www.qiushibaike.com'
    start_urls = ['https://www.qiushibaike.com/text/page/1/']

    def parse(self, response):
        # print(response) # encoding text 经过解码后的 body 没有解码 xpath css 
        content = response.xpath("//div[contains(@class, 'old-style-col1')]/div")
        # xpath extract extract_first get = extract getall 
        for dz in content:
            author = dz.xpath(".//h2/text()").get().strip()
            context = dz.xpath(".//div[@class='content']//text()").getall()
            context = ''.join(context).strip()
            item = QsbkItem(author=author, context=context)
            yield dict(item)
        next_url = response.xpath("//ul[@class='pagination']/li[last()]/a/@href").get()
        if not next_url:
            return
        else:
            yield scrapy.Request(self.base_domain + next_url, callback=self.parse)


